package com.push.spider.spider;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.push.common.config.MinioConfig;
import com.push.common.config.RuoYiConfig;
import com.push.common.config.TransConfig;
import com.push.common.manager.BeanManager;
import com.push.common.utils.*;
import com.push.common.utils.httpclient.HttpClientResult;
import com.push.common.utils.httpclient.HttpClientUtil;
import com.push.framework.web.domain.server.Sys;
import com.push.oss.utils.AttachmentHelper;
import com.push.spider.utils.UrlUtils;
import com.push.web.domain.*;
import com.push.web.service.*;
import com.push.common.constant.FileTypeConstants;
import lombok.SneakyThrows;
import org.jruby.RubyBoolean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.FileInputStream;
import java.util.*;

public class HttpClientSpider {
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private ArrayList<String> hasDownUrls = new ArrayList<>();

    private Queue<String> queue = new LinkedList<String>();

    private BizSource source;

    private List<BizLabelSecond> labelSeconds = new ArrayList<>();

    @Autowired
    IBizArticleService articleService = BeanManager.getBean(IBizArticleService.class);

    @Autowired
    IBizSourceService sourceService = BeanManager.getBean(IBizSourceService.class);

    @Autowired
    IBizLabelSecondService labelSecondService = BeanManager.getBean(IBizLabelSecondService.class);

    @Autowired
    IBizArticleAppendixService articleAppendixService = BeanManager.getBean(IBizArticleAppendixService.class);

    @Autowired
    MinioUtil minioUtil = BeanManager.getBean(MinioUtil.class);

    @Autowired
    MinioConfig prop = BeanManager.getBean(MinioConfig.class);

    @Autowired
    IBizMultimodalFileService multimodalFileService = BeanManager.getBean(IBizMultimodalFileService.class);

    @Autowired
    private AttachmentHelper attachmentHelper;
    private HttpServletRequest request = null;

    @Autowired
    private TransConfig transConfig = BeanManager.getBean(TransConfig.class);

    public int size = 0;

    public HttpClientSpider(String sourceId) {
        this.source = sourceService.selectBizSourceBySourceId(Long.valueOf(sourceId));
        this.labelSeconds = labelSecondService.selectBizLabelSecondListByParentIds(this.source.getLabelIds());
    }

    /**
     * 指定爬虫的初始url，可实现：爬取用户指定页面，
     * 使用方法参考：MyTaskOneTime.ClientSpider()
     */
    public void SetSourceSpiderUrlWithUserProvicedUrl(String userSpiderUrl){
        this.source.setSpiderUrl(userSpiderUrl);
    }

    private JSONObject articleInfo = new JSONObject();

    @SneakyThrows
    public void run(){
        //查重策略：先获取本地所有datakey，避免每次访问数据库
        List<String> dataKeyList = articleService.selectBizDataKey();
        Set<String> articleDataKeyHashSet = new HashSet<>(dataKeyList);
        List<String> multimodalDataKeyList = multimodalFileService.selectBizDataKey();
        Set<String> multimodalFileDataKeyHashSet = new HashSet<>(multimodalDataKeyList);

        queue.offer(source.getSpiderUrl());
        int CNT = 1;
        boolean test_flag = false;
        do{
            String url = queue.poll();
            //调试+测试
            //String url = "https://cloud.kepuchina.cn/newSearch/videoDetail?id=6964342635592908800";
            //System.out.println("url:"+url);
            if (hasDownUrls.contains(url)){
                //if(test_flag){break;}
                continue;
            }
            System.out.println("获取url:"+url);
            HttpClientResult personResult = new HttpClientResult();
            if (url.startsWith("https")){
                personResult = HttpClientUtil.doGet(url, true);
            } else {
                personResult = HttpClientUtil.doGet(url);
            }
            hasDownUrls.add(url);
            size++;
            Html html = new Html(personResult.getContent());
            if (StringUtils.regex(url,source.getPageXpath())) {
                System.out.println("-----列表页-----");
                //列表页
                //对于从来没有检索的数据源，或者10天内没有检索过的数据源，遍历全部页码，否则，只检索前2页
                if (StringUtils.isNotEmpty(source.getPageDivXpath())){
                    List<String> links = html.xpath(source.getPageDivXpath()).links().all();
                    //System.out.println("下一页：" + links.toString());
                    for (String link : links) {
                        // queue.offer(UrlUtils.buildUrl(url,link));
                        String pageUrl = UrlUtils.buildUrl(url,link);
                        //跳过已爬取过的url
                        if (hasDownUrls.contains(pageUrl)){
                            continue;
                        }
                        //System.out.println("路由url:" + link + ";  " + UrlUtils.buildUrl(url,link));
                        queue.offer(pageUrl);
                    }
                }
                // 如果需要爬取详情页，则继续把详情页的链接添加到爬虫队列中。
                // 如还有其他列表需要抓取，则继续添加
                //获取列表
                int articleCount = 0;   //列表获取的文章计数
                if (!StringUtils.isEmpty(source.getListXpath())) {
                    List<String> listXpaths = Arrays.asList(source.getListXpath().split(";"));
                    for (String listXpath:listXpaths){
                        List<Selectable> tagNodes = html.xpath(listXpath).nodes();
                        if (tagNodes.size() == 0){
                            System.out.println(personResult.getContent());
                            System.out.println("列表未找到节点");
                        }
                        for (Selectable contentNode : tagNodes) {
                            String titleOrg = extractInfo(source.getTitleXpath(), contentNode);
                            String sourceUrl = UrlUtils.buildUrl(url, extractInfo(source.getDetailsXpath(), contentNode));

                            if (StringUtils.isEmpty(sourceUrl)){
                                continue;
                            }
                            //检查全文sourceUrl是否已获取并保存入库
                            // boolean isSaved = articleService.checkArticleIsSaved(sourceUrl);
                            //if (isSaved) {
                            String dataKey = EncryptUtil.encrypt(sourceUrl, "md5");
                            if(articleDataKeyHashSet.contains(dataKey)){
                                System.out.println("文章已获取过，跳过：" + sourceUrl);
                                continue;   //文章已经爬取，跳过
                            }
                            System.out.println("列表获取新文章链接：" + sourceUrl);

                            String avator = UrlUtils.buildUrl(url,extractInfo(source.getAvatorXpath(), contentNode));
                            if (source.getPicDownload().equalsIgnoreCase("是")){
                                System.out.println("列表逻辑存图片：" + avator);
                                //这里会出现重复保存avator到minio的情况，后续待完善处理
                                avator = UrlUtils.downloadFileToMinio(avator, minioUtil);
                            }
                            String publishTime = extractInfo(source.getPublishTimeXpath(), contentNode);

                            queue.offer(sourceUrl);  //添加到队列
                            articleCount += 1;
                            //String dataKey = EncryptUtil.encrypt(sourceUrl, "md5");
                            BizArticle bizArticle = buildArticle(dataKey, titleOrg, sourceUrl, avator, publishTime, source.getSourceId());
                            System.out.println(bizArticle.toString());
                            BizArticle bizArticleDb = articleService.selectBizArticleByDataKey(dataKey);
                            if (StringUtils.isNull(bizArticleDb)){
                                if (strContains(bizArticle.getTitleOrg(), labelSeconds)){
                                    //自动翻译标题
                                    if (StringUtils.isContainsChinese(bizArticle.getTitleOrg())){
                                        bizArticle.setTitleCn(bizArticle.getTitleOrg());
                                    } else {
                                        String transWord = getTransResult(bizArticle.getTitleOrg(), "auto", "zh");
                                        JSONObject resultObj = JSONObject.parseObject(transWord);
                                        JSONArray jsonArray = resultObj.getJSONArray("trans_result");
                                        if (StringUtils.isNotNull(jsonArray) && jsonArray.size() > 0){
                                            bizArticle.setTitleCn(jsonArray.getJSONObject(0).getString("dst"));
                                        }
                                    }
                                    //自动翻译摘要
                                    if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(bizArticle.getSummaryOrg())){
                                        String transWord = getTransResult(bizArticle.getSummaryOrg(), "auto", "zh");
                                        JSONObject titleObj = JSONObject.parseObject(transWord);
                                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                                        if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                                            bizArticle.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                                        }
                                    }
                                    articleService.insertBizArticle(bizArticle);
                                }
                            } else {
                                bizArticle.setId(bizArticleDb.getId());
                                articleService.updateBizArticle(bizArticle);
                            }
                            //暂存文章信息(后续存储关联的多模态数据需要访问)
                            articleInfo.put(dataKey, bizArticle);
                        }
                    }
                    System.out.println("获取文章详情链接数量：" + articleCount + ", 列表：" + url);
                }
            } else {
                System.out.println("=====详情页=====");
                // 如果有业务需要抓取详情页，则继续在这儿抓取。
                String dataKey = EncryptUtil.encrypt(url, "md5");
                //String summaryOrg = UrlUtils.repairContent(extractInfo(source.getSummaryXpath(), html), url);
                //String contentsOrg = UrlUtils.repairContent(extractInfo(source.getContentsXpath(), html), url);
                String summaryOrg = extractInfo(source.getSummaryXpath(), html);
                String contentsOrg = extractInfo(source.getContentsXpath(), html);
                System.out.println("全文内容：" + contentsOrg);
                if (source.getPicDownload().equalsIgnoreCase("是")){
                    BizArticle bizArticle = new BizArticle();
                    if(articleInfo.containsKey(dataKey)) {
                        bizArticle = (BizArticle) articleInfo.getObject(dataKey, BizArticle.class);
                    }else{
                        bizArticle = articleService.selectBizArticleByDataKey(dataKey);
                        if(bizArticle == null) {
                            System.out.println("填充文章测试数据");
                            bizArticle.setUrl(url);
                            bizArticle.setTitleOrg("");
                        }else{
                            System.out.println("查库结果：" + bizArticle.toString());
                        }
                    }
                    //1. 抽取和保存全文中的图片
                    List<String> imgUrls = UrlUtils.praseUrlFromString(summaryOrg, url);
                    for(int k = 0; k < imgUrls.size(); k++){
                        String mDataKey = EncryptUtil.encrypt(imgUrls.get(k), "md5");
                        if(multimodalFileDataKeyHashSet.contains(mDataKey)){
                            imgUrls.remove(k);
                            k--;    // 因为删除元素后，后面的元素会向前移动，所以需要将下标减一
                        }else{
                            //缓存，用于下次判断
                            multimodalFileDataKeyHashSet.add(mDataKey);
                        }
                    }
                    if (imgUrls.size() > 0){
                        System.out.println("SummaryXpath_Find Image:" + imgUrls.size());
                        //List<String> newImages = UrlUtils.downloadFilesToMinio(imgUrls, minioUtil);
                        JSONObject newImagesInfoObj = UrlUtils.downloadFilesToMinioV2(imgUrls, minioUtil);
                        List<String> newImages = new ArrayList<>();
                        List<String> newImagesSizeList = new ArrayList<>();
                        JSONArray newUrls = newImagesInfoObj.getJSONArray("newUrls");
                        JSONArray fileSizeList = newImagesInfoObj.getJSONArray("fileSizeList");
                        for (int i = 0; i < newUrls.size(); i++) {
                            newImages.add(newUrls.getString(i));
                            newImagesSizeList.add(fileSizeList.getString(i));
                        }

                        summaryOrg = UrlUtils.repairDownloadContent(summaryOrg, newImages);

                        //202404（摘要）图片信息存入本地数据库：dataType(0图片1视频2音频3文本)，图片原始url列表，图片本地path列表，文章url，文章标题(作为图片name)，文章key，数据源Id
                        //saveFileInfoToDbByBatch(FileTypeConstants.FILE_IMAGE, imgUrls, newImages, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId());
                        multimodalFileDataKeyHashSet = saveFileInfoToDbByBatchV2(FileTypeConstants.FILE_IMAGE, imgUrls, newImages, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId(), newImagesSizeList, multimodalFileDataKeyHashSet);
                    }
                    List<String> contentImgUrls = UrlUtils.praseUrlFromString(contentsOrg, url);
                    //System.out.println("图片url修复：" +contentImgUrls.toString());
                    if (contentImgUrls.size() > 0){
                        System.out.println("ContentsXpath_Find Image:" + contentImgUrls.size());
                        //List<String> newImages = UrlUtils.downloadFilesToMinio(contentImgUrls, minioUtil);
                        JSONObject newImagesInfoObj = UrlUtils.downloadFilesToMinioV2(contentImgUrls, minioUtil);
                        List<String> newImages = new ArrayList<>();
                        List<String> newImagesSizeList = new ArrayList<>();
                        JSONArray newUrls = newImagesInfoObj.getJSONArray("newUrls");
                        JSONArray fileSizeList = newImagesInfoObj.getJSONArray("fileSizeList");
                        for (int i = 0; i < newUrls.size(); i++) {
                            newImages.add(newUrls.getString(i));
                            newImagesSizeList.add(fileSizeList.getString(i));
                        }
                        contentsOrg = UrlUtils.repairDownloadContent(contentsOrg, newImages);

                        //System.out.println("test1:" + newImages.toString());
                        //System.out.println("test2:" + newImagesSizeList.toString());
                        //202404（全文）图片信息存入本地数据库：dataType(0图片1视频2音频3文本)，图片原始url列表，图片本地path列表，文章url，文章标题(作为图片name)，文章key，数据源Id
                        //saveFileInfoToDbByBatch(FileTypeConstants.FILE_IMAGE, contentImgUrls, newImages, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId());
                        multimodalFileDataKeyHashSet = saveFileInfoToDbByBatchV2(FileTypeConstants.FILE_IMAGE, contentImgUrls, newImages, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId(), newImagesSizeList, multimodalFileDataKeyHashSet);
                    }

                    //2.抽取和保存全文中视频
                    String patternStr="<source\\s*([^>]*)\\s*src=\\\"(.*?)\\\"\\s*([^>]*)>";
                    List<String> contentVideoUrls = UrlUtils.praseObjUrlFromString(contentsOrg, url, patternStr);
                    System.out.println("视频资源：" + contentVideoUrls);
                    if (contentVideoUrls.size() > 0){
                        System.out.println("ContentsXpath_Find Video:" + contentVideoUrls.size());
                        //List<String> newVideo = UrlUtils.downloadFilesToMinio(contentVideoUrls, minioUtil);
                        ////202404（全文）视频信息存入本地数据库：dataType(0图片1视频2音频3文本)，数据原始url列表，数据本地path列表，文章url，文章标题(作为图片name)，文章key，数据源Id
                        //saveFileInfoToDbByBatch(FileTypeConstants.FILE_VIDEO, contentVideoUrls, newVideo, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId());

                        //只保存第一个视频（其他是重复的）。检查文件是否已保存入库
                        //boolean isSaved = multimodalFileService.checkFileIsSaved(contentVideoUrls.get(0));
                        //if(!isSaved) {
                        String mDataKey = EncryptUtil.encrypt(contentVideoUrls.get(0), "md5");
                        if(!multimodalFileDataKeyHashSet.contains(mDataKey)){
                            //String newVideo = UrlUtils.downloadFileToMinio(contentVideoUrls.get(0), minioUtil);
                            List<String> newVideoListInfo = UrlUtils.downloadFileToMinioV2(contentVideoUrls.get(0), minioUtil);
                            String newVideo = newVideoListInfo.get(0);
                            System.out.println("视频保存成功：" + newVideo);
                            List<String> newVideoList = Arrays.asList(newVideo);
                            contentsOrg = UrlUtils.repairDownloadObjContent(contentsOrg, newVideoList, patternStr);
                            // dataType(0图片1视频2音频3文本)，头像本地存储地址，图片原始url，图片本地path，文章url，文章标题(作为图片name)，文章key，数据源Id
                            System.out.println("视频入库..");
                            String fileSize = newVideoListInfo.get(1);
                            saveFileInfoToDb(FileTypeConstants.FILE_VIDEO, "", contentVideoUrls.get(0), newVideo, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId(), fileSize);
                            System.out.println("视频入库成功！");
                            //缓存，用于下次判断
                            multimodalFileDataKeyHashSet.add(mDataKey);
                        }
                    }

                    //3.抽取和保存全文中音频
                    //System.out.println("test0:" + html);
                    //System.out.println("test1:" + source.getContentsXpath());
                    //System.out.println("test2:" + contentsOrg);
                    patternStr="<audio\\s*([^>]*)\\s*src=\\\"(.*?)\\\"\\s*([^>]*)>";
                    List<String> contentAudioUrls = UrlUtils.praseObjUrlFromString(contentsOrg, url, patternStr);
                    System.out.println("音频资源：" + contentAudioUrls);
                    if (contentAudioUrls.size() > 0){
                        System.out.println("ContentsXpath_Find Audio:" + contentAudioUrls.size());
                        //List<String> newAudio = UrlUtils.downloadFilesToMinio(contentAudioUrls, minioUtil);
                        ////202404（全文）视频信息存入本地数据库：dataType(0图片1视频2音频3文本)，数据原始url列表，数据本地path列表，文章url，文章标题(作为图片name)，文章key，数据源Id
                        //saveFileInfoToDbByBatch(FileTypeConstants.FILE_AUDIO, contentAudioUrls, newAudio, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId());

                        //只保存第一个视频（其他是重复的）。检查文件是否已保存入库
                        //boolean isSaved = multimodalFileService.checkFileIsSaved(contentAudioUrls.get(0));
                        //if(!isSaved) {
                        String mDataKey = EncryptUtil.encrypt(contentAudioUrls.get(0), "md5");
                        if(!multimodalFileDataKeyHashSet.contains(mDataKey)){
                            //String newAudio = UrlUtils.downloadFileToMinio(contentAudioUrls.get(0), minioUtil);
                            List<String> newAudioListInfo = UrlUtils.downloadFileToMinioV2(contentAudioUrls.get(0), minioUtil);
                            String newAudio = newAudioListInfo.get(0);
                            System.out.println("音频保存成功：" + newAudio);
                            List<String> newAudioList = Arrays.asList(newAudio);
                            contentsOrg = UrlUtils.repairDownloadObjContent(contentsOrg, newAudioList, patternStr);
                            // dataType(0图片1视频2音频3文本)，头像本地存储地址，图片原始url，图片本地path，文章url，文章标题(作为图片name)，文章key，数据源Id
                            System.out.println("音频入库..");
                            String fileSize = newAudioListInfo.get(1);
                            saveFileInfoToDb(FileTypeConstants.FILE_AUDIO, "", contentAudioUrls.get(0), newAudio, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId(), fileSize);
                            System.out.println("音频入库成功！");
                            //缓存，用于下次判断
                            multimodalFileDataKeyHashSet.add(mDataKey);
                        }
                    }

                } else {
                    summaryOrg = UrlUtils.repairContent(summaryOrg, url);
                    contentsOrg = UrlUtils.repairContent(contentsOrg, url);
                }

                //修改发布机构支持添加多个xpath，用;号隔开
                //String publisher = extractInfo(source.getPublisherXpath(), html);
                String publisher = "";
                List<String> publisherXpathList = Arrays.asList(source.getPublisherXpath().split(";"));
                if(publisherXpathList.size() > 0){
                    for (String publisherXpath: publisherXpathList){
                        publisher = extractInfo(publisherXpath, html).trim();
                        if(!publisher.isEmpty()){
                            break;
                        }
                    }
                }
                System.out.println("发布机构：" + publisher);
                String publishTime = extractInfo(source.getPublishTimeXpath(), html);
                if(StringUtils.isNotEmpty(publisher)){
                    publisher = publisher.replaceAll("来源：","");
                }
                if(StringUtils.isNotEmpty(publishTime)){
                    publishTime = publishTime.replaceAll("发布时间：","");
                }

                // 取全文的中第一张图作为文章的avator
                String avator = "";
                List<String> contentXpathList = Arrays.asList(source.getContentsXpath().split(";"));
                if(contentXpathList.size() > 0){
                    for (String contentXpath: contentXpathList){
                        List<Selectable> contentNodes = html.xpath(contentXpath).nodes();
                        if(contentNodes.size() > 0){
                            /*
                             * 单独获取图片
                             */
                            List<Selectable> imgNodes = contentNodes.get(0).xpath("//img/@src").nodes();
                            if (imgNodes.size() > 0) {
                                avator = UrlUtils.buildUrl(url, imgNodes.get(0).toString());
                                if (source.getPicDownload().equalsIgnoreCase("是")){
                                    // 这里重复将avator图片保存到minio（后续考虑去重）
                                    avator = UrlUtils.downloadFileToMinio(avator, minioUtil);
                                    // avator = downloadFileToOss(avator, "image");    //OSS存储不成功，需调试
                                }
                            }
                            break;
                        }
                    }
                }
                /*if (source.getPicDownload().equalsIgnoreCase("是")){
                    //avator = UrlUtils.downloadFileToMinio(avator, minioUtil);
                    avator = downloadFileToOss(avator, "image");
                    System.out.println("test2-下载图片2:" + avator);
                }*/
                //更新文章内容
                BizArticle bizArticle = updateArticle(dataKey, summaryOrg, contentsOrg, publisher, publishTime, avator, source.getSourceId());
                System.out.println("新增或更新全文：" + bizArticle.toString());
                BizArticle bizArticleDb = articleService.selectBizArticleByDataKey(dataKey);
                if (StringUtils.isNull(bizArticleDb)){
                    if (strContains(bizArticle.getTitleOrg(), labelSeconds)){
                        articleService.insertBizArticle(bizArticle);
                    }
                } else {
                    bizArticle.setId(bizArticleDb.getId());
                    if (StringUtils.isNotEmpty(bizArticleDb.getPublishTime())){
                        bizArticle.setPublishTime(bizArticleDb.getPublishTime());
                    }
                    if (StringUtils.isNotEmpty(bizArticleDb.getPublishFormatTime())){
                        bizArticle.setPublishFormatTime(bizArticleDb.getPublishFormatTime());
                    }
                    //自动翻译摘要
                    if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(bizArticle.getSummaryOrg())){
                        String transWord = getTransResult(bizArticle.getSummaryOrg(), "auto", "zh");
                        JSONObject titleObj = JSONObject.parseObject(transWord);
                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                        if (titleArray.size() > 0){
                            bizArticle.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                        }
                    }/* else if (StringUtils.isNotEmpty(article.getContentsCn())){
                        List<String> sentenceList = HanLP.extractSummary(article.getContentsCn(),10);
                        if(sentenceList.size() > 0){
                            String abstact = sentenceList.toString();
                            article.setSummaryOrg(abstact.substring(1,abstact.lastIndexOf("]")));
                            String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                            JSONObject titleObj = JSONObject.parseObject(transWord);
                            JSONArray titleArray = titleObj.getJSONArray("trans_result");
                            if (titleArray.size() > 0){
                                article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                            }
                        }
                    }*/
                    articleService.updateBizArticle(bizArticle);
                }
                if(StringUtils.isNotEmpty(source.getFilesXpath())){
                    List<String> filesXpathList = Arrays.asList(source.getFilesXpath().split(";"));
                    List<BizArticleAppendix> appendices = new ArrayList<>();
                    for (String fileXpath: filesXpathList){
                        List<Selectable> fileNodes = html.xpath(fileXpath).nodes();
                        if(fileNodes.size() > 0){
                            for (Selectable fileNode:fileNodes){
                                BizArticleAppendix appendix = new BizArticleAppendix();
                                appendix.setAppendixUrl(fileNode.toString());
                                appendix.setArticleId(bizArticle.getId());
                            }
                        }
                    }
                    if (!appendices.isEmpty()){
                        articleAppendixService.batchSave(appendices);
                    }
                }

                CNT += 1;
                if(CNT>1 && test_flag){System.out.println("测试：退出系统！");break;}
            }
            Thread.sleep(source.getIntervalNum()*1000);
        } while (queue.size() > 0);
    }

    public String extractInfo(String xpathString, Selectable contentNode){
        if (StringUtils.isEmpty(xpathString)){
            return "";
        }
        List<String> xpathList = Arrays.asList(xpathString.split(";"));
        for (String xpath: xpathList){
            List<Selectable> nodes = contentNode.xpath(xpath).nodes();
            if(nodes.size() > 0){
                return nodes.get(0).toString();
            }
        }
        return "";
    }

    public BizArticle buildArticle(String dataKey, String titleOrg, String url,String avator, String publishTime, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setTitleOrg(titleOrg);
        article.setUrl(url);
        article.setAvator(avator);
        article.setPublishTime(publishTime);
        article.setPublishFormatTime(DateUtils.formatDate(DateUtils.parseDate(publishTime),publishTime));
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public BizArticle updateArticle(String dataKey, String summaryOrg, String contentsOrg, String publisher, String publishTime, String avator, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setSummaryOrg(summaryOrg);
        contentsOrg.replaceAll("visibility: hidden;", "visibility: visible;");
        article.setContentsOrg(contentsOrg);
        article.setContentsCn(StringUtils.delHtmlTags(contentsOrg));
        article.setPublisher(publisher);
        article.setPublishTime(publishTime);
        article.setPublishFormatTime(DateUtils.formatDate(DateUtils.parseDate(publishTime),publishTime));
        article.setAvator(avator);
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public static boolean strContains(String str, List<BizLabelSecond> labels){
        if (labels == null){
            return true;
        }
        if (labels.isEmpty()){
            return true;
        }
        if(StringUtils.isEmpty(str)){
            return false;
        }
        long len = labels.parallelStream().filter(label -> str.toLowerCase(Locale.ROOT).contains(label.getLabelName().toLowerCase(Locale.ROOT))).count();
        if (len > 0){
            return true;
        }
        return false;
    }

    public String getTransResult(String query, String from, String to) {
        Map<String, String> params = buildParams(query, from, to);
        return HttpClientUtil.doGet(transConfig.getTransUrl(), params,false).getContent();
    }

    private Map<String, String> buildParams(String query, String from, String to) {
        Map<String, String> params = new HashMap<String, String>();
        params.put("q", query);
        params.put("from", from);
        params.put("to", to);
        params.put("appid", transConfig.getAppid());
        // 随机数
        String salt = String.valueOf(System.currentTimeMillis());
        params.put("salt", salt);
        // 签名
        String src = transConfig.getAppid() + query + salt + transConfig.getSecurityKey();
        params.put("sign", MD5.md5(src));
        return params;
    }

    //多模态数据：dataType(0图片1视频2音频3文本)，头像本地存储地址，图片原始url，图片本地path，文章url，文章标题(作为图片name)，文章key，数据源Id
    private int saveFileInfoToDb(int dataType, String avator, String origUrl, String savedUrl, String articleUrl, String articleTitle, String articleDataKey, long sourceId, String fileSize) {
        String dataKey = EncryptUtil.encrypt(origUrl, "md5");

        //保存文件描述到表biz_multimodal_file
        BizMultimodalFile bizMultimodalFile = new BizMultimodalFile();
        bizMultimodalFile.setName(articleTitle.isEmpty() ? "NULL" : articleTitle);
        bizMultimodalFile.setDataType(dataType);
        bizMultimodalFile.setDataKey(dataKey);
        bizMultimodalFile.setAvator(avator);
        bizMultimodalFile.setSummaryOrg("");   //TODO：可获取文件标题描述
        bizMultimodalFile.setUrl(origUrl);
        bizMultimodalFile.setLocalPath(savedUrl);
        bizMultimodalFile.setArticleUrl(articleUrl);
        bizMultimodalFile.setLabels("");      //文章标签在后期维护中更新
        bizMultimodalFile.setArticle(articleDataKey);
        bizMultimodalFile.setSourceId(sourceId);
        bizMultimodalFile.setSourceType(0);
        bizMultimodalFile.setUploadUserId((long)-1);

        //文件后缀
        String fileExtend = UrlUtils.getFileExtendByName(origUrl);
        bizMultimodalFile.setFileFormat(fileExtend);
        //文件大小
        if(fileSize.isEmpty()) {
            System.out.println("单独计算文件大小..");
            fileSize = UrlUtils.getFileSizeByUrlWithFormat(savedUrl);
        }
        bizMultimodalFile.setSize(fileSize);

        return multimodalFileService.insertBizMultimodalFile(bizMultimodalFile);
    }
    // 批量保存图片描述信息到本地数据库
    // FileTypeConstants.FILE_IMAGE, List<String> imgUrls, List<String> newImages, bizArticle.getUrl(), bizArticle.getTitleOrg(), dataKey, source.getSourceId());
    private int saveFileInfoToDbByBatch(int dataType, List<String> imgUrls, List<String> newImages, String articleUrl, String articleTitle, String articleDataKey, long sourceId){
        for (int i=0; i<imgUrls.size(); i++){
            //检查文件是否已保存入库
            boolean isSaved = multimodalFileService.checkFileIsSaved(imgUrls.get(i));
            if(!isSaved) {
                // dataType(0图片1视频2音频3文本)，头像本地存储地址，图片原始url，图片本地path，文章url，文章标题(作为图片name)，文章key，数据源Id
                String fileSize = "";
                saveFileInfoToDb(dataType, "", imgUrls.get(i), newImages.get(i), articleUrl, articleTitle, articleDataKey, sourceId, fileSize);
            }
        }
        return 0;
    }
    private Set<String> saveFileInfoToDbByBatchV2(int dataType, List<String> imgUrls, List<String> newImages, String articleUrl, String articleTitle, String articleDataKey, long sourceId, List<String> fileSizeList, Set<String> multimodalFileDataKeyHashSet){
        for (int i=0; i<imgUrls.size(); i++) {
            //检查文件是否已保存入库
            //boolean isSaved = multimodalFileService.checkFileIsSaved(imgUrls.get(i));
            //if(!isSaved) {
            if (newImages.get(i) != null && !newImages.get(i).isEmpty()) {
                String mDataKey = EncryptUtil.encrypt(imgUrls.get(i), "md5");
                if (!multimodalFileDataKeyHashSet.contains(mDataKey)) {
                    // dataType(0图片1视频2音频3文本)，头像本地存储地址，图片原始url，图片本地path，文章url，文章标题(作为图片name)，文章key，数据源Id
                    saveFileInfoToDb(dataType, "", imgUrls.get(i), newImages.get(i), articleUrl, articleTitle, articleDataKey, sourceId, fileSizeList.get(i));
                    multimodalFileDataKeyHashSet.add(mDataKey);
                }
            }
        }
        return multimodalFileDataKeyHashSet;
    }

    private String downloadFileToOss(String fileUrl, String dir){
        String filePath = HttpClientUtil.downLoadFromUrl(fileUrl, SnowflakeIdWorker.generateId()+ LinkHelper.getTypeByExtenssion(fileUrl), RuoYiConfig.getProfile());
        File file = new File(filePath);
        try {
            MultipartFile cMultiFile = new MockMultipartFile("file", file.getName(), UrlUtils.getContentType(filePath), new FileInputStream(file));
            return attachmentHelper.upload(request, cMultiFile, dir);
        }catch (Exception e) {
            System.out.println("保存文件到OSS错误：" + e.getMessage());
            e.printStackTrace();
            return "";
        }
    }

    public int getSize(){
        return size;
    }
}
